package jsi.spider

import com.thoughtworks.xstream.XStreamimport jsi.spider.WpArtimport org.apache.commons.logging.Logimport org.apache.commons.logging.LogFactory;

class GSpider {

	
	List toProcess
	List processed
//	List wpArts 
	def urlBase = 'http://wiadomosci.wp.pl/'
	Log log = LogFactory.getLog(this.getClass())
	XStream xstream = new XStream()
	
	Date startDate = new Date()
	def startProcessedSize
	def startToProcessSize
	
	static void main(args) {
		def spider = new GSpider()
		
		spider.init()
		
		spider.go()
		
		spider.writeLists()
		
	//	createUrlFile()
	}
	///html/body/div[8]/div/div/h1
	
	def go(){
		
		printFiguresStart()
		while (toProcess.size() > 0 && runningSec() < 240){
			processFirst(toProcess, processed)
		}
		printFiguresEnd()
	}
	
	def init(){
		toProcess = xstream.fromXML(new FileInputStream("etc/wp_articles/toProcess.xml"))
	//	processed = new ArrayList<WpArt>();
		processed = xstream.fromXML(new FileInputStream("etc/wp_articles/processed.xml"))
		startProcessedSize = processed.size()
		startToProcessSize = toProcess.size()
	//	wpArts = xstream.fromXML(new FileInputStream("etc/wp_articles/articles.xml"))
	}
	
	def processFirst(toProcess, processed){
		def wpArt = toProcess.get(0)
		log.debug wpArt
		def parser = new org.cyberneko.html.parsers.SAXParser()
		
		parser.setFeature('http://xml.org/sax/features/namespaces', false)
		def page = new XmlParser(parser).parse(wpArt.url)
		
		page.depthFirst().DIV.each{
			if (it.'@id' == 'bxMr'){
				it.depthFirst().A.each{
					addIfNew(toProcess, processed, new WpArt(
							url:urlBase+it.'@href',
							title:it.'@value'
					))
				}
			}
		}
		page.depthFirst().SPAN.each{
			if (it.'@id' == 'adv_wl'){
					wpArt.text = it.text()
					it.depthFirst().STRONG.each{
						wpArt.addStrong(it.text())
					}
					it.depthFirst().B.each{
						wpArt.addB(it.text())
					}
				}
		}
		
		processed.add(wpArt)
		toProcess.remove(wpArt)
	}
	
	def writeLists(){
		XStream xstream = new XStream();
		xstream.toXML(toProcess, new FileOutputStream("etc/wp_articles/toProcess.xml"));
		xstream.toXML(processed, new FileOutputStream("etc/wp_articles/processed.xml"));
	}
	
	def static createUrlFile(){
		XStream xstream = new XStream();
		List urlList = new ArrayList<String>();
		urlList.add(
			new WpArt(url:"http://wiadomosci.wp.pl/wiadomosc.html?kat=1342&wid=10052225&ticaid=160fd&_ticrsn=3")
		)
				
		xstream.toXML(urlList, new FileOutputStream("etc/wp_articles/toProcess.xml"));
		xstream.toXML(urlList, new FileOutputStream("etc/wp_articles/processed.xml"));
		List<WpArt> artList = new ArrayList<WpArt>()
		xstream.toXML(artList, new FileOutputStream("etc/wp_articles/articles.xml"));
	}
	
	def addIfNew(toProcess, processed, wpArt){
		for (WpArt saved:toProcess){
			if (saved.url == wpArt.url){
				log.debug "already in queue " + wpArt.url
				return
			}
		}
		
		for (WpArt saved:processed){
			if (saved.url == wpArt.url){
				log.debug "already processed " + wpArt.url
				return
			}
		}
		
		log.debug "adding to queue " + wpArt.url
		toProcess.add(wpArt)
	}
	
	def runningSec(){
		Date now = new Date();
		long time = (new Date().getTime() - startDate.getTime())/1000
		return time
	}
	
	def printFiguresStart(){
		log.debug "start " + startDate
		log.debug "toProcess.size: " + toProcess.size() 
		log.debug "processed.size: " + processed.size() 
	}
	
	def printFiguresEnd(){
		log.debug "end " + new Date() + " time " + runningSec()
		log.debug "toProcess.size: " + toProcess.size()
		log.debug "processed.size: " + processed.size()
		log.debug "processed growth: " + (processed.size() - startProcessedSize)
		log.debug "toProcess growth: " + (toProcess.size() - startToProcessSize)
	}
}